# -*- coding: utf-8 -*-
'''
Created on 04-11-2012

@author: LONG HOANG GIANG
'''


import os
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
from CrawlerLib import commonlib, html2text
from Model.storymodelv2 import StoryModel
import datetime
import workerpool
import re
from pprint import pprint



class FourVN():
    
    startPage = 1
    urlfm = 'http://4vn.eu/forum/showthread.php?t={0}&page={1}'
    sid = ''
    listUrl = []
    
    def __init__(self, sid, startPage=1):
        self.startPage = startPage
        self.url = self.urlfm.format(sid, startPage)
        self.sid = sid
        
    def extractTitle2(self, headerNode):
        commonlib.Etree.clean_elements(headerNode, 'font', {'size': 6})
        commonlib.Etree.clean_elements(headerNode, 'font', {'size': 3})
        commonlib.Etree.clean_elements(headerNode, 'font', {'size': 5})
        html = commonlib.Etree.tostring(headerNode)
        
        pattern = ["(Chương[^<]+)", "(Giới Thiệu[^<]*)", "(chương[^<]+)", "(CHƯƠNG[^<]+)", 
                   "(Chap[^<]+)", "(chap[^<]+)", "(Văn Án[^<]+)", "(văn án[^<]+)"]
        
        for pat in pattern:
            title = commonlib.extractText(r"{0}".format(pat), html, 1, '')
            if title != '': break
        
        title = re.sub(r"Chương (\d+)", lambda m: "Chương {0}".format(int(float(m.group(1)))), title)
        return title.strip()
    
    def extractTitle(self, headerNode):
        commonlib.Etree.clean_elements(headerNode, 'font', {'size': 6})
        commonlib.Etree.clean_elements(headerNode, 'font', {'size': 3})
        commonlib.Etree.clean_elements(headerNode, 'font', {'size': 5})
        title = html2text.html2text(commonlib.Etree.tostring(headerNode), textonly=True)
        pattern = ["(Chương.+)", "(Giới Thiệu.*)", "(Chuơng.+)", "(\d+ *: *.+)", "(chương.+)", "(Văn Án.*)", "(Chap.+)", "(chap.+)"]
        for ititle in title.split("\n"):
            ititle = ititle.strip()
            if ititle == '': continue
            ok = False
            for pat in pattern:
                if re.search(r"{0}".format(pat).decode('utf-8'), ititle):
                    title = ititle
                    ok = True
                    break
            if ok: break
            else: title = ''
        
        if not 'Chương' in title and not 'Chuơng' in title and not 'chương' in title and not 'Chap' in title:
            if re.search(r"(\d+ *: *.+)", title):
                title = re.sub(r"(\d+ *: *.+)", "Chương \\1", title)
        title = re.sub(unicode("Chương\s+(\d+)(.*)"), lambda m: "Chương {0}{1}".format(int(float(m.group(1))), m.group(2)), title)
        title = re.sub(unicode("Chuơng\s+(\d+)(.*)"), lambda m: "Chương {0}{1}".format(int(float(m.group(1))), m.group(2)), title)
        title = re.sub(unicode("chương\s+(\d+)(.*)"), lambda m: "Chương {0}{1}".format(int(float(m.group(1))), m.group(2)), title)
        title = re.sub(unicode("Chap\s+(\d+)(.*)"), lambda m: "Chương {0}{1}".format(int(float(m.group(1))), m.group(2)), title)
        return title.strip()

    def processPage(self, url):
        tree = commonlib.loadweb(url).build_tree()
        data = []
        for node in tree.xpath("//div[contains(@id, 'post_message_')]"):
#            print commonlib.Etree.tostring(node)
            hienraNode = node.xpath(".//input[@value='Hiện ra']/../following-sibling::*[1]".decode('utf-8'))
            if len(hienraNode) > 0:
                content = html2text.html2text(commonlib.Etree.tostring(hienraNode)).strip()
                content = re.sub(ur"( *\n *){3,}", "\n\n", content)
                content = re.sub(ur"\n", "<br />", content)
                hienraNode[0].getparent().remove(hienraNode[0])
                titleTxt = html2text.html2text(commonlib.Etree.tostring(node))
                chapter = commonlib.toUpper(re.sub(ur" *: *", ": ", commonlib.extractText(ur"(Chương *\d+[^\n]+)", titleTxt, 1)))
                chapter = re.sub(ur"<.+?>", '', chapter)
                chapter = re.sub(ur": *\?* *$", "", chapter).strip()
                if chapter == "": chapter = "Đang cập nhật"
                print chapter
                content = "<b>{0}</b><br /><hr /><br />{1}".format(chapter, content)
                data.append({'title': chapter, 'content': content})
            else:
                content = html2text.html2text(commonlib.Etree.tostring(node)).strip()
                chapter = commonlib.toUpper(re.sub(ur" *: *", ": ", commonlib.extractText(ur"(Chương *\d+[^\n]+)", content, 1)))
                chapter = re.sub(ur"<.+?>", '', chapter)
                chapter = re.sub(ur": *\?* *$", "", chapter).strip()
                content = re.sub(ur"( *\n *){3,}", "\n\n", content)
                content = re.sub(ur"\n", "<br />", content)
                content = "<b>{0}</b><br /><hr /><br />{1}".format(chapter, content)
                print chapter
                data.append({'title': chapter, 'content': content})
        return data
            
    def buildListUrl(self):
        self.listUrl = []
        for ipage in range(self.startPage, self.detectLastPage(self.url) + 1):
            self.listUrl.append(self.urlfm.format(self.sid, ipage))
    
    def preloadAndCache(self):
        
        def loadWeb(url):
            commonlib.loadweb(url)
        self.buildListUrl()
        pool = workerpool.WorkerPool(size=10)
        pool.map(loadWeb, self.listUrl)
        pool.shutdown()
        pool.wait()
        
    def detectLastPage(self, url):
        tree = commonlib.loadweb(url).build_tree()
        lastPageNode = tree.xpath("//div[@class='pagenav']/table//td/a[contains(., 'Last')]")
        if len(lastPageNode) > 0:
            ahref = lastPageNode[0].get('href')
            lastpage = int(commonlib.extractText(r"page=(\d+)", ahref, 1, 0))
            return lastpage
        return 0
    
    def process(self):
        print '-------------------- LOAD AND CACHE ---------------------------'
        self.preloadAndCache()
        print '-------------------- START NOW ---------------------------'
        smodel = StoryModel("/longhoanggiang/database/{0}".format(DATABASENAME))
        smodel.open(True)
        for url in self.listUrl:
            data = self.processPage(url)
            for item in data:
                if item['title'].strip() == '': continue
                
                print "-----------------------------"
                print "TITLE: {0}".format(item['title'])
                print "CONTENT: {0}\n{1}\n".format(len(item['content']), item['content'][:200])
                print "-----------------------------"
                
                smodel.add_story(commonlib.toUpper(item['title']), item['content'], 0)
        smodel.close()
        

if __name__ == '__main__':
    
    DATABASENAME = 'luumanhlsu'
    
    a = FourVN('63782')
    a.process()
#    aa = a.processPage('http://4vn.eu/forum/showthread.php?t=63782&page=122')
#    for item in aa:
#        print '#########################################################'
#        print item['title']
#        print item['content']
#        print '#########################################################'
    
#    

        
    
    print '> Finished at {0}'.format(datetime.datetime.now())
    os._exit(1)
    
